import random
import time
from selenium import webdriver
from scrapy import  Selector
import csv

browser = webdriver.Chrome()

# browser.get("http://fangzi.xmfish.com")
with open('同安.csv', 'w', encoding="utf-8", newline="") as f:
    header = ["title", "href", 'infos', 'addr', 'more_info', 'price', 'type', 'update_time']
    writer = csv.DictWriter(f, fieldnames=header)
    writer.writeheader()
    for i in range(1,5):
        url = "http://fangzi.xmfish.com/web/search_hire.html?h=&hf=&ca=59206&r=&s=&a=&rm=&f=&d=&tp=&l=0&tg=&hw=&o=&ot=0&tst=0&page={}".format(str(i))
        browser.get(url)
        time.sleep(random.randint(3, 5))
        browser.implicitly_wait(10)
        html = Selector(text=browser.page_source)
        lis = html.xpath('//*[@id="select_tab2"]/ul/li')
        for li in lis:
            house_info = {}
            title = li.xpath('./div[3]/h3/a/text() | ./div[2]/h3/a/text()').extract()[-1]
            title = "".join([i.strip() for i in title if len(i.strip()) > 0])
            href = "http://fangzi.xmfish.com"+li.xpath('./div[3]/h3/a/@href | ./div[2]/h3/a/@href').extract_first() #http://fangzi.xmfish.com/
            infos = li.xpath('./div[3]/span[1]//text()|./div[2]/span[1]//text()').extract()
            infos = "=".join([i.strip() for i in infos if len(i.strip())>0])
            addr = li.xpath('./div[3]/span[2]//text() | ./div[2]/span[2]//text()').extract()
            addr = "".join([i.strip() for i in addr])
            more_info = li.xpath('./div[3]/div//text() | ./div[2]/div//text()').extract()
            more_info = "=".join([i.strip() for i in more_info if len(i.strip())>0])
            price = li.xpath('./div[3]/span[3]/b/text()|./div[2]/span[3]/b/text()').extract_first()
            type = li.xpath('./div[3]/span[4]/text() | ./div[2]/span[4]/text()').extract_first()
            update_time = li.xpath('./div[3]/span[5]/text() |./div[2]/span[5]/text()').extract_first()
            house_info['title'] = title
            house_info['href'] = href
            house_info['infos'] = infos
            house_info['addr'] = addr
            house_info['more_info'] = more_info
            if price:
                price = int(price)
            else:
                price = 0
            house_info['price'] = price
            house_info['type'] = type
            house_info['update_time'] = update_time
            print(house_info)
            writer.writerow(house_info)
        time.sleep(random.randint(3, 5))
        browser.implicitly_wait(10)


